﻿using System;
using System.Collections.Generic;
using System.Collections;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Serialization;
using System.IO;

namespace PreProcessingLanguage
{
    public class PreProcessing
    {

        Dictionary<string, int> demTu = new Dictionary<string, int>();

        public static string DAUCAU = "[.,;“”:()!?'`\\\"/] ?";
        public static string TUTHUA = @"\[title\]|\[\/title\]|\[content\]|\[\/content\]|&quot|&gt|Url:|Title:|Tom tat:|Noi dung:";

        public PreProcessing()
        {
        }

        public Dictionary<string, int> DemTu { get { return demTu; } set { demTu = value; } }
        public string xoaTuThua(string input)
        {
            return Regex.Replace(input, TUTHUA, "", RegexOptions.Compiled);
        }
        public string xoaSo(string input)
        {
            return Regex.Replace(input, @"\d+", "", RegexOptions.Compiled);
        }
        public void xuLy1(string input)
        {
            input = xoaSo(xoaTuThua(input));
            foreach (string sen in Regex.Split(input, DAUCAU))
            {
                MatchCollection s2 = Regex.Matches(sen, @"\w+");
                foreach (Match word in s2)
                {
                    if (demTu.ContainsKey(word.Value.ToLower()))
                    {
                        demTu[word.Value.ToLower()] += 1;
                    }
                    else
                    {
                        demTu[word.Value.ToLower()] = 1;
                    }
                }
            }

        }
        public void xuly2(string input)
        {
            input = xoaSo(xoaTuThua(input));
            string word = "";
            foreach (string sen in Regex.Split(input, DAUCAU))
            {
                MatchCollection s2 = Regex.Matches(sen, @"\w+");
                for (int i = 0; i < s2.Count - 1; i++)
                {
                    word = (s2[i] + " " + s2[i + 1]).ToLower();
                    if (demTu.ContainsKey(word))
                    {
                        demTu[word] += 1;
                    }
                    else
                    {
                        demTu[word] = 1;
                    }
                }
            }

        }
        public void xuly3(string input)
        {
            input = xoaSo(xoaTuThua(input));
            string word = "";
            foreach (string sen in Regex.Split(input, DAUCAU))
            {
                MatchCollection s2 = Regex.Matches(sen, @"\w+");
                for (int i = 0; i < s2.Count - 2; i++)
                {
                    word = (s2[i] + " " + s2[i + 1] + " " + s2[i + 2]).ToLower();
                    if (demTu.ContainsKey(word))
                    {
                        demTu[word] += 1;
                    }
                    else
                    {
                        demTu[word] = 1;
                    }
                }
            }

        }
        public static string ConvertToUnSign(string text)
        {
            Regex regex = new Regex(@"\p{IsCombiningDiacriticalMarks}+");
            string strFormD = text.Normalize(System.Text.NormalizationForm.FormD);
            return regex.Replace(strFormD, String.Empty).Replace('\u0111', 'd').Replace('\u0110', 'D');
        }

        public void writeData(string path)
        {
            StreamWriter sw = new StreamWriter(path);
            foreach (KeyValuePair<string, int> pair in demTu)
            {
                if(pair.Value>5)
                    sw.Write(pair.Key + ":" + pair.Value + "\n");
            }
            sw.Flush();
            sw.Close();
        }
        public void readData(string path)
        {
            StreamReader sr = new StreamReader(path);
            string s;
            while ((s = sr.ReadLine()) != null)
            {
                string[] kv = s.Split(':');
                demTu[kv[0]] = Int32.Parse(kv[1]);
            }
            sr.Close();
        }
        public void getCount()
        {
            Console.WriteLine("Tong so: " + demTu.Count);
        }
        public void clear()
        {
            demTu.Clear();
        }
    }
}
